{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:01:05.292164Z",
     "start_time": "2020-02-24T14:01:05.290042Z"
    }
   },
   "outputs": [],
   "source": [
    "# old script to convert the data to hdf5\n",
    "#!python tmp/helmi2000-all-to-hdf5.py raw.hdf5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:30:38.783038Z",
     "start_time": "2020-02-24T14:30:38.780770Z"
    }
   },
   "outputs": [],
   "source": [
    "import vaex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:30:41.260950Z",
     "start_time": "2020-02-24T14:30:41.219012Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                                    </th><th>E              </th><th>FeH                </th><th>L                 </th><th>Lz                 </th><th>id  </th><th>vx         </th><th>vy         </th><th>vz         </th><th>x          </th><th>y          </th><th>z           </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>        </td><td>-143642.46875  </td><td>-0.8500438566576901</td><td>530.4976196289062 </td><td>494.7076110839844  </td><td>0   </td><td>88.4153824 </td><td>49.6388206 </td><td>-31.668335 </td><td>2.02209926 </td><td>-4.46000433</td><td>1.2478652   </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>        </td><td>-147369.71875  </td><td>-1.2243692168671811</td><td>486.2104797363281 </td><td>455.4730224609375  </td><td>0   </td><td>-15.5355339</td><td>-146.026901</td><td>-3.09953618</td><td>-2.81293559</td><td>2.8778379  </td><td>-1.10256541 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>        </td><td>-135592.3125   </td><td>-0.719111582147683 </td><td>767.2000122070312 </td><td>523.798095703125   </td><td>0   </td><td>82.9779053 </td><td>-107.426102</td><td>101.553047 </td><td>-5.4028182 </td><td>0.68217665 </td><td>0.0781376138</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>        </td><td>-156535.640625 </td><td>-1.0243906881991864</td><td>467.8145446777344 </td><td>414.2808532714844  </td><td>0   </td><td>245.805817 </td><td>-33.8533859</td><td>-76.2258148</td><td>1.24775958 </td><td>-1.85724545</td><td>0.25229013  </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>        </td><td>-144798.59375  </td><td>-0.6298426574045468</td><td>615.1791381835938 </td><td>533.921875         </td><td>0   </td><td>-104.228561</td><td>35.5588379 </td><td>61.1774826 </td><td>1.64664936 </td><td>4.56083202 </td><td>-0.714936256</td></tr>\n",
       "<tr><td>...                                  </td><td>...            </td><td>...                </td><td>...               </td><td>...                </td><td>... </td><td>...        </td><td>...        </td><td>...        </td><td>...        </td><td>...        </td><td>...         </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,995</i></td><td>-140634.03125  </td><td>-1.784470129811317 </td><td>223.27337646484375</td><td>-165.4918670654297 </td><td>32  </td><td>45.4977951 </td><td>7.44010353 </td><td>-38.8004951</td><td>1.60329461 </td><td>3.89954162 </td><td>-3.06202364 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,996</i></td><td>-125076.1875   </td><td>-1.6468850848866747</td><td>323.4261779785156 </td><td>-283.8592834472656 </td><td>32  </td><td>-66.9358826</td><td>-112.574234</td><td>-60.1542549</td><td>4.44476748 </td><td>3.23454309 </td><td>2.96048737  </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,997</i></td><td>-127123.4765625</td><td>-1.9548544460061874</td><td>384.3844909667969 </td><td>-329.7651672363281 </td><td>32  </td><td>-100.639084</td><td>1.37331498 </td><td>-50.4876251</td><td>-5.87667322</td><td>-3.19651794</td><td>-1.85363591 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,998</i></td><td>-123743.21875  </td><td>-1.8058613773294687</td><td>425.5377502441406 </td><td>-337.26629638671875</td><td>32  </td><td>11.1622276 </td><td>119.667099 </td><td>-84.9928894</td><td>-2.31978846</td><td>5.34516335 </td><td>-2.76076818 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,999</i></td><td>-132432.21875  </td><td>-1.5350944349862055</td><td>455.4076843261719 </td><td>-252.4229278564453 </td><td>32  </td><td>109.544586 </td><td>184.828583 </td><td>146.21405  </td><td>-2.71977496</td><td>-2.2846334 </td><td>-0.70853442 </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#          E                FeH                  L                   Lz                   id    vx           vy           vz           x            y            z\n",
       "0          -143642.46875    -0.8500438566576901  530.4976196289062   494.7076110839844    0     88.4153824   49.6388206   -31.668335   2.02209926   -4.46000433  1.2478652\n",
       "1          -147369.71875    -1.2243692168671811  486.2104797363281   455.4730224609375    0     -15.5355339  -146.026901  -3.09953618  -2.81293559  2.8778379    -1.10256541\n",
       "2          -135592.3125     -0.719111582147683   767.2000122070312   523.798095703125     0     82.9779053   -107.426102  101.553047   -5.4028182   0.68217665   0.0781376138\n",
       "3          -156535.640625   -1.0243906881991864  467.8145446777344   414.2808532714844    0     245.805817   -33.8533859  -76.2258148  1.24775958   -1.85724545  0.25229013\n",
       "4          -144798.59375    -0.6298426574045468  615.1791381835938   533.921875           0     -104.228561  35.5588379   61.1774826   1.64664936   4.56083202   -0.714936256\n",
       "...        ...              ...                  ...                 ...                  ...   ...          ...          ...          ...          ...          ...\n",
       "3,299,995  -140634.03125    -1.784470129811317   223.27337646484375  -165.4918670654297   32    45.4977951   7.44010353   -38.8004951  1.60329461   3.89954162   -3.06202364\n",
       "3,299,996  -125076.1875     -1.6468850848866747  323.4261779785156   -283.8592834472656   32    -66.9358826  -112.574234  -60.1542549  4.44476748   3.23454309   2.96048737\n",
       "3,299,997  -127123.4765625  -1.9548544460061874  384.3844909667969   -329.7651672363281   32    -100.639084  1.37331498   -50.4876251  -5.87667322  -3.19651794  -1.85363591\n",
       "3,299,998  -123743.21875    -1.8058613773294687  425.5377502441406   -337.26629638671875  32    11.1622276   119.667099   -84.9928894  -2.31978846  5.34516335   -2.76076818\n",
       "3,299,999  -132432.21875    -1.5350944349862055  455.4076843261719   -252.4229278564453   32    109.544586   184.828583   146.21405    -2.71977496  -2.2846334   -0.70853442"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = vaex.open('raw.hdf5')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:30:42.768931Z",
     "start_time": "2020-02-24T14:30:42.762312Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "E      float64\n",
       "FeH    float64\n",
       "L      float64\n",
       "Lz     float64\n",
       "id       int64\n",
       "vx     float64\n",
       "vy     float64\n",
       "vz     float64\n",
       "x      float64\n",
       "y      float64\n",
       "z      float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:30:43.581544Z",
     "start_time": "2020-02-24T14:30:43.570042Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "290400000"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.nbytes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:30:44.067610Z",
     "start_time": "2020-02-24T14:30:44.055798Z"
    }
   },
   "outputs": [],
   "source": [
    "# we write them out for the order\n",
    "column_names = 'id x y z vx vy vz E L Lz'.split()\n",
    "df = df[column_names]\n",
    "for name in column_names:\n",
    "    if name == 'id':\n",
    "        to_type = 'uint8'\n",
    "    else:\n",
    "        to_type = 'float32'\n",
    "    df[name] = df[name].astype(to_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:30:46.285301Z",
     "start_time": "2020-02-24T14:30:46.027320Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                                    </th><th>id  </th><th>x                  </th><th>y                  </th><th>z                    </th><th>vx                 </th><th>vy                 </th><th>vz                 </th><th>E              </th><th>L                 </th><th>Lz                 </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>        </td><td>25  </td><td>1.5591524839401245 </td><td>-1.0355582237243652</td><td>-1.234255075454712   </td><td>-333.9376220703125 </td><td>14.448123931884766 </td><td>-127.04698181152344</td><td>-120484.9921875</td><td>706.5674438476562 </td><td>-323.2850341796875 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>        </td><td>0   </td><td>-1.2007783651351929</td><td>-0.4624782204627991</td><td>-0.35920777916908264 </td><td>251.57025146484375 </td><td>-230.97308349609375</td><td>-57.01335906982422 </td><td>-155375.984375 </td><td>428.27996826171875</td><td>393.6932373046875  </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>        </td><td>7   </td><td>-1.7470163106918335</td><td>1.9494433403015137 </td><td>-2.0347394943237305  </td><td>-300.6556396484375 </td><td>209.7186279296875  </td><td>98.73015594482422  </td><td>-91698.1484375 </td><td>1023.0885620117188</td><td>219.72926330566406 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>        </td><td>4   </td><td>-5.907489776611328 </td><td>-1.205744743347168 </td><td>3.528154134750366    </td><td>-157.84805297851562</td><td>-189.96688842773438</td><td>200.09796142578125 </td><td>-79208.953125  </td><td>1201.3680419921875</td><td>931.9030151367188  </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>        </td><td>25  </td><td>-4.605956554412842 </td><td>-2.8173961639404297</td><td>1.4604418277740479   </td><td>-19.823850631713867</td><td>41.93743133544922  </td><td>5.556484699249268  </td><td>-143025.40625  </td><td>260.6395568847656 </td><td>-249.0136260986328 </td></tr>\n",
       "<tr><td>...                                  </td><td>... </td><td>...                </td><td>...                </td><td>...                  </td><td>...                </td><td>...                </td><td>...                </td><td>...            </td><td>...               </td><td>...                </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,995</i></td><td>16  </td><td>-10.562079429626465</td><td>-13.517659187316895</td><td>-0.255374014377594   </td><td>113.22222137451172 </td><td>15.439544677734375 </td><td>-27.641550064086914</td><td>-82553.796875  </td><td>1454.4359130859375</td><td>1367.4256591796875 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,996</i></td><td>23  </td><td>4.051898956298828  </td><td>1.5419585704803467 </td><td>-1.059655785560608   </td><td>-223.50880432128906</td><td>109.15838623046875 </td><td>-91.93414306640625 </td><td>-120274.9375   </td><td>995.6219482421875 </td><td>786.9400634765625  </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,997</i></td><td>22  </td><td>2.541799306869507  </td><td>1.946831464767456  </td><td>-1.0098917484283447  </td><td>107.93811798095703 </td><td>33.78214645385742  </td><td>-84.91947937011719 </td><td>-157932.25     </td><td>209.9375          </td><td>-124.26988983154297</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,998</i></td><td>27  </td><td>6.1693034172058105 </td><td>4.346409320831299  </td><td>7.26422643661499     </td><td>33.274295806884766 </td><td>-72.5516128540039  </td><td>-84.66383361816406 </td><td>-102009.875    </td><td>979.6717529296875 </td><td>-592.2166137695312 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,999</i></td><td>22  </td><td>0.10937126725912094</td><td>-1.9420192241668701</td><td>-0.029312405735254288</td><td>-66.17059326171875 </td><td>202.93203735351562 </td><td>114.3353271484375  </td><td>-168679.609375 </td><td>241.0592041015625 </td><td>-106.30963134765625</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#          id    x                    y                    z                      vx                   vy                   vz                   E                L                   Lz\n",
       "0          25    1.5591524839401245   -1.0355582237243652  -1.234255075454712     -333.9376220703125   14.448123931884766   -127.04698181152344  -120484.9921875  706.5674438476562   -323.2850341796875\n",
       "1          0     -1.2007783651351929  -0.4624782204627991  -0.35920777916908264   251.57025146484375   -230.97308349609375  -57.01335906982422   -155375.984375   428.27996826171875  393.6932373046875\n",
       "2          7     -1.7470163106918335  1.9494433403015137   -2.0347394943237305    -300.6556396484375   209.7186279296875    98.73015594482422    -91698.1484375   1023.0885620117188  219.72926330566406\n",
       "3          4     -5.907489776611328   -1.205744743347168   3.528154134750366      -157.84805297851562  -189.96688842773438  200.09796142578125   -79208.953125    1201.3680419921875  931.9030151367188\n",
       "4          25    -4.605956554412842   -2.8173961639404297  1.4604418277740479     -19.823850631713867  41.93743133544922    5.556484699249268    -143025.40625    260.6395568847656   -249.0136260986328\n",
       "...        ...   ...                  ...                  ...                    ...                  ...                  ...                  ...              ...                 ...\n",
       "3,299,995  16    -10.562079429626465  -13.517659187316895  -0.255374014377594     113.22222137451172   15.439544677734375   -27.641550064086914  -82553.796875    1454.4359130859375  1367.4256591796875\n",
       "3,299,996  23    4.051898956298828    1.5419585704803467   -1.059655785560608     -223.50880432128906  109.15838623046875   -91.93414306640625   -120274.9375     995.6219482421875   786.9400634765625\n",
       "3,299,997  22    2.541799306869507    1.946831464767456    -1.0098917484283447    107.93811798095703   33.78214645385742    -84.91947937011719   -157932.25       209.9375            -124.26988983154297\n",
       "3,299,998  27    6.1693034172058105   4.346409320831299    7.26422643661499       33.274295806884766   -72.5516128540039    -84.66383361816406   -102009.875      979.6717529296875   -592.2166137695312\n",
       "3,299,999  22    0.10937126725912094  -1.9420192241668701  -0.029312405735254288  -66.17059326171875   202.93203735351562   114.3353271484375    -168679.609375   241.0592041015625   -106.30963134765625"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.sample(frac=1, random_state=42)  # shuffle\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:30:53.689535Z",
     "start_time": "2020-02-24T14:30:52.040397Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[#####################################---] 96.06% estimated time:     0.05s =  0.0m =  0.0h  "
     ]
    }
   ],
   "source": [
    "df.export('helmi-dezeeuw-2000-FeH-v2-old.hdf5', progress=True, column_names=column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:30:58.107501Z",
     "start_time": "2020-02-24T14:30:58.058067Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                                    </th><th>id  </th><th>x                  </th><th>y                  </th><th>z                    </th><th>vx                 </th><th>vy                 </th><th>vz                 </th><th>E              </th><th>L                 </th><th>Lz                 </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>        </td><td>25  </td><td>1.5591524839401245 </td><td>-1.0355582237243652</td><td>-1.234255075454712   </td><td>-333.9376220703125 </td><td>14.448123931884766 </td><td>-127.04698181152344</td><td>-120484.9921875</td><td>706.5674438476562 </td><td>-323.2850341796875 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>        </td><td>0   </td><td>-1.2007783651351929</td><td>-0.4624782204627991</td><td>-0.35920777916908264 </td><td>251.57025146484375 </td><td>-230.97308349609375</td><td>-57.01335906982422 </td><td>-155375.984375 </td><td>428.27996826171875</td><td>393.6932373046875  </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>        </td><td>7   </td><td>-1.7470163106918335</td><td>1.9494433403015137 </td><td>-2.0347394943237305  </td><td>-300.6556396484375 </td><td>209.7186279296875  </td><td>98.73015594482422  </td><td>-91698.1484375 </td><td>1023.0885620117188</td><td>219.72926330566406 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>        </td><td>4   </td><td>-5.907489776611328 </td><td>-1.205744743347168 </td><td>3.528154134750366    </td><td>-157.84805297851562</td><td>-189.96688842773438</td><td>200.09796142578125 </td><td>-79208.953125  </td><td>1201.3680419921875</td><td>931.9030151367188  </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>        </td><td>25  </td><td>-4.605956554412842 </td><td>-2.8173961639404297</td><td>1.4604418277740479   </td><td>-19.823850631713867</td><td>41.93743133544922  </td><td>5.556484699249268  </td><td>-143025.40625  </td><td>260.6395568847656 </td><td>-249.0136260986328 </td></tr>\n",
       "<tr><td>...                                  </td><td>... </td><td>...                </td><td>...                </td><td>...                  </td><td>...                </td><td>...                </td><td>...                </td><td>...            </td><td>...               </td><td>...                </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,995</i></td><td>16  </td><td>-10.562079429626465</td><td>-13.517659187316895</td><td>-0.255374014377594   </td><td>113.22222137451172 </td><td>15.439544677734375 </td><td>-27.641550064086914</td><td>-82553.796875  </td><td>1454.4359130859375</td><td>1367.4256591796875 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,996</i></td><td>23  </td><td>4.051898956298828  </td><td>1.5419585704803467 </td><td>-1.059655785560608   </td><td>-223.50880432128906</td><td>109.15838623046875 </td><td>-91.93414306640625 </td><td>-120274.9375   </td><td>995.6219482421875 </td><td>786.9400634765625  </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,997</i></td><td>22  </td><td>2.541799306869507  </td><td>1.946831464767456  </td><td>-1.0098917484283447  </td><td>107.93811798095703 </td><td>33.78214645385742  </td><td>-84.91947937011719 </td><td>-157932.25     </td><td>209.9375          </td><td>-124.26988983154297</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,998</i></td><td>27  </td><td>6.1693034172058105 </td><td>4.346409320831299  </td><td>7.26422643661499     </td><td>33.274295806884766 </td><td>-72.5516128540039  </td><td>-84.66383361816406 </td><td>-102009.875    </td><td>979.6717529296875 </td><td>-592.2166137695312 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,999</i></td><td>22  </td><td>0.10937126725912094</td><td>-1.9420192241668701</td><td>-0.029312405735254288</td><td>-66.17059326171875 </td><td>202.93203735351562 </td><td>114.3353271484375  </td><td>-168679.609375 </td><td>241.0592041015625 </td><td>-106.30963134765625</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#          id    x                    y                    z                      vx                   vy                   vz                   E                L                   Lz\n",
       "0          25    1.5591524839401245   -1.0355582237243652  -1.234255075454712     -333.9376220703125   14.448123931884766   -127.04698181152344  -120484.9921875  706.5674438476562   -323.2850341796875\n",
       "1          0     -1.2007783651351929  -0.4624782204627991  -0.35920777916908264   251.57025146484375   -230.97308349609375  -57.01335906982422   -155375.984375   428.27996826171875  393.6932373046875\n",
       "2          7     -1.7470163106918335  1.9494433403015137   -2.0347394943237305    -300.6556396484375   209.7186279296875    98.73015594482422    -91698.1484375   1023.0885620117188  219.72926330566406\n",
       "3          4     -5.907489776611328   -1.205744743347168   3.528154134750366      -157.84805297851562  -189.96688842773438  200.09796142578125   -79208.953125    1201.3680419921875  931.9030151367188\n",
       "4          25    -4.605956554412842   -2.8173961639404297  1.4604418277740479     -19.823850631713867  41.93743133544922    5.556484699249268    -143025.40625    260.6395568847656   -249.0136260986328\n",
       "...        ...   ...                  ...                  ...                    ...                  ...                  ...                  ...              ...                 ...\n",
       "3,299,995  16    -10.562079429626465  -13.517659187316895  -0.255374014377594     113.22222137451172   15.439544677734375   -27.641550064086914  -82553.796875    1454.4359130859375  1367.4256591796875\n",
       "3,299,996  23    4.051898956298828    1.5419585704803467   -1.059655785560608     -223.50880432128906  109.15838623046875   -91.93414306640625   -120274.9375     995.6219482421875   786.9400634765625\n",
       "3,299,997  22    2.541799306869507    1.946831464767456    -1.0098917484283447    107.93811798095703   33.78214645385742    -84.91947937011719   -157932.25       209.9375            -124.26988983154297\n",
       "3,299,998  27    6.1693034172058105   4.346409320831299    7.26422643661499       33.274295806884766   -72.5516128540039    -84.66383361816406   -102009.875      979.6717529296875   -592.2166137695312\n",
       "3,299,999  22    0.10937126725912094  -1.9420192241668701  -0.029312405735254288  -66.17059326171875   202.93203735351562   114.3353271484375    -168679.609375   241.0592041015625   -106.30963134765625"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = vaex.open('./helmi-dezeeuw-2000-FeH-v2-old.hdf5')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:30:59.978095Z",
     "start_time": "2020-02-24T14:30:59.967354Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "116.44363403320312"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.nbytes / 1024**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:22:58.563922Z",
     "start_time": "2020-02-24T14:22:58.222220Z"
    }
   },
   "outputs": [],
   "source": [
    "df_10p = df.sample(frac=0.1, random_state=42)\n",
    "df_10p.export('helmi-dezeeuw-2000-FeH-v2-10percent.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:23:09.224482Z",
     "start_time": "2020-02-24T14:23:09.181213Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                                  </th><th>id  </th><th>x                   </th><th>y                   </th><th>z                   </th><th>vx                 </th><th>vy                 </th><th>vz                 </th><th>E             </th><th>L                 </th><th>Lz                 </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>      </td><td>0   </td><td>1.2318683862686157  </td><td>-0.39692866802215576</td><td>-0.598057746887207  </td><td>301.1552734375     </td><td>174.05947875976562 </td><td>27.42754554748535  </td><td>-149431.40625 </td><td>407.38897705078125</td><td>333.9555358886719  </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>      </td><td>23  </td><td>-0.16370061039924622</td><td>3.654221296310425   </td><td>-0.25490644574165344</td><td>-195.00022888183594</td><td>170.47216796875    </td><td>142.5302276611328  </td><td>-124247.953125</td><td>890.2411499023438 </td><td>684.6676025390625  </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>      </td><td>32  </td><td>-2.120255947113037  </td><td>3.326052665710449   </td><td>1.7078403234481812  </td><td>-48.63423156738281 </td><td>171.6472930908203  </td><td>-2.079437255859375 </td><td>-138500.546875</td><td>372.2410888671875 </td><td>-202.17617797851562</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>      </td><td>8   </td><td>4.7155890464782715  </td><td>4.5852508544921875  </td><td>2.2515437602996826  </td><td>-232.42083740234375</td><td>-294.850830078125  </td><td>62.85865020751953  </td><td>-60037.0390625</td><td>1297.63037109375  </td><td>-324.6875          </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>      </td><td>16  </td><td>7.21718692779541    </td><td>11.99471664428711   </td><td>-1.064562201499939  </td><td>-1.6891745328903198</td><td>181.329345703125   </td><td>-11.333610534667969</td><td>-83206.84375  </td><td>1332.7989501953125</td><td>1328.948974609375  </td></tr>\n",
       "<tr><td>...                                </td><td>... </td><td>...                 </td><td>...                 </td><td>...                 </td><td>...                </td><td>...                </td><td>...                </td><td>...           </td><td>...               </td><td>...                </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,995</i></td><td>21  </td><td>1.9938701391220093  </td><td>0.789276123046875   </td><td>0.22205990552902222 </td><td>-216.92990112304688</td><td>16.124420166015625 </td><td>-211.244384765625  </td><td>-146457.4375  </td><td>457.72247314453125</td><td>203.36758422851562 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,996</i></td><td>25  </td><td>3.7180912494659424  </td><td>0.721337616443634   </td><td>1.6415337324142456  </td><td>-185.92160034179688</td><td>-117.25082397460938</td><td>-105.4986572265625 </td><td>-126627.109375</td><td>335.0025634765625 </td><td>-301.8370056152344 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,997</i></td><td>14  </td><td>0.3688507676124573  </td><td>13.029608726501465  </td><td>-3.633934736251831  </td><td>-53.677146911621094</td><td>-145.15771484375   </td><td>76.70909881591797  </td><td>-84912.2578125</td><td>817.1375732421875 </td><td>645.8507080078125  </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,998</i></td><td>18  </td><td>-0.11259264498949051</td><td>1.4529125690460205  </td><td>2.168952703475952   </td><td>179.30865478515625 </td><td>205.79710388183594 </td><td>-68.75872802734375 </td><td>-133498.46875 </td><td>724.000244140625  </td><td>-283.6910400390625 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,999</i></td><td>4   </td><td>20.796220779418945  </td><td>-3.331387758255005  </td><td>12.18841552734375   </td><td>42.69000244140625  </td><td>69.20479583740234  </td><td>29.54275131225586  </td><td>-65519.328125 </td><td>1843.07470703125  </td><td>1581.4151611328125 </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#        id    x                     y                     z                     vx                   vy                   vz                   E               L                   Lz\n",
       "0        0     1.2318683862686157    -0.39692866802215576  -0.598057746887207    301.1552734375       174.05947875976562   27.42754554748535    -149431.40625   407.38897705078125  333.9555358886719\n",
       "1        23    -0.16370061039924622  3.654221296310425     -0.25490644574165344  -195.00022888183594  170.47216796875      142.5302276611328    -124247.953125  890.2411499023438   684.6676025390625\n",
       "2        32    -2.120255947113037    3.326052665710449     1.7078403234481812    -48.63423156738281   171.6472930908203    -2.079437255859375   -138500.546875  372.2410888671875   -202.17617797851562\n",
       "3        8     4.7155890464782715    4.5852508544921875    2.2515437602996826    -232.42083740234375  -294.850830078125    62.85865020751953    -60037.0390625  1297.63037109375    -324.6875\n",
       "4        16    7.21718692779541      11.99471664428711     -1.064562201499939    -1.6891745328903198  181.329345703125     -11.333610534667969  -83206.84375    1332.7989501953125  1328.948974609375\n",
       "...      ...   ...                   ...                   ...                   ...                  ...                  ...                  ...             ...                 ...\n",
       "329,995  21    1.9938701391220093    0.789276123046875     0.22205990552902222   -216.92990112304688  16.124420166015625   -211.244384765625    -146457.4375    457.72247314453125  203.36758422851562\n",
       "329,996  25    3.7180912494659424    0.721337616443634     1.6415337324142456    -185.92160034179688  -117.25082397460938  -105.4986572265625   -126627.109375  335.0025634765625   -301.8370056152344\n",
       "329,997  14    0.3688507676124573    13.029608726501465    -3.633934736251831    -53.677146911621094  -145.15771484375     76.70909881591797    -84912.2578125  817.1375732421875   645.8507080078125\n",
       "329,998  18    -0.11259264498949051  1.4529125690460205    2.168952703475952     179.30865478515625   205.79710388183594   -68.75872802734375   -133498.46875   724.000244140625    -283.6910400390625\n",
       "329,999  4     20.796220779418945    -3.331387758255005    12.18841552734375     42.69000244140625    69.20479583740234    29.54275131225586    -65519.328125   1843.07470703125    1581.4151611328125"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = vaex.open('./helmi-dezeeuw-2000-FeH-v2-10percent.hdf5')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:24:06.118164Z",
     "start_time": "2020-02-24T14:24:06.109478Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11.644363403320312"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.nbytes/1024**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:24:41.417815Z",
     "start_time": "2020-02-24T14:24:41.294602Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                                 </th><th>id  </th><th>x                   </th><th>y                   </th><th>z                      </th><th>vx                 </th><th>vy                 </th><th>vz                 </th><th>E             </th><th>L                 </th><th>Lz               </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>     </td><td>0   </td><td>1.2318683862686157  </td><td>-0.39692866802215576</td><td>-0.598057746887207     </td><td>301.1552734375     </td><td>174.05947875976562 </td><td>27.42754554748535  </td><td>-149431.40625 </td><td>407.38897705078125</td><td>333.9555358886719</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>     </td><td>0   </td><td>3.1000826358795166  </td><td>-5.2641072273254395 </td><td>-0.08563928306102753   </td><td>42.31336975097656  </td><td>117.65007019042969 </td><td>-48.13563919067383 </td><td>-135327.78125 </td><td>660.099853515625  </td><td>587.467041015625 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>     </td><td>0   </td><td>2.239809036254883   </td><td>3.041299819946289   </td><td>0.20514149963855743    </td><td>-238.2696533203125 </td><td>-33.5640754699707  </td><td>-12.8736572265625  </td><td>-137704.390625</td><td>650.582275390625  </td><td>649.4723510742188</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>     </td><td>0   </td><td>0.8771036863327026  </td><td>1.7859472036361694  </td><td>0.930344820022583      </td><td>-186.66604614257812</td><td>-15.906522750854492</td><td>-57.79670715332031 </td><td>-167917.625   </td><td>353.51385498046875</td><td>319.4240417480469</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>     </td><td>0   </td><td>0.1723162978887558  </td><td>3.2632522583007812  </td><td>1.307965874671936      </td><td>-94.61146545410156 </td><td>33.2225341796875   </td><td>-45.685874938964844</td><td>-158379.78125 </td><td>386.50628662109375</td><td>314.4658508300781</td></tr>\n",
       "<tr><td>...                               </td><td>... </td><td>...                 </td><td>...                 </td><td>...                    </td><td>...                </td><td>...                </td><td>...                </td><td>...           </td><td>...               </td><td>...              </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>10,009</i></td><td>0   </td><td>0.90480637550354    </td><td>4.832354545593262   </td><td>-0.08294452726840973   </td><td>-96.9912109375     </td><td>18.704917907714844 </td><td>-22.30633544921875 </td><td>-149742.75    </td><td>497.9065246582031 </td><td>485.6202392578125</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>10,010</i></td><td>0   </td><td>0.5669021606445312  </td><td>1.3759788274765015  </td><td>0.7074300646781921     </td><td>-178.99111938476562</td><td>292.1639404296875  </td><td>15.780367851257324 </td><td>-143447.40625 </td><td>471.4540100097656 </td><td>411.9163513183594</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>10,011</i></td><td>0   </td><td>-0.26697027683258057</td><td>2.6641666889190674  </td><td>0.6911221742630005     </td><td>-129.47744750976562</td><td>-153.0867919921875 </td><td>-191.64637756347656</td><td>-139728.703125</td><td>576.6127319335938 </td><td>385.8191223144531</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>10,012</i></td><td>0   </td><td>-2.1996543407440186 </td><td>-0.6662538051605225 </td><td>-0.00011480561806820333</td><td>109.60547637939453 </td><td>-59.14215087890625 </td><td>-8.455219268798828 </td><td>-182279.234375</td><td>204.04580688476562</td><td>203.1173553466797</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>10,013</i></td><td>0   </td><td>0.15803414583206177 </td><td>-3.096275806427002  </td><td>0.15831845998764038    </td><td>124.51082611083984 </td><td>174.66983032226562 </td><td>-95.10889434814453 </td><td>-148366.75    </td><td>493.0278015136719 </td><td>413.1236572265625</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#       id    x                     y                     z                        vx                   vy                   vz                   E               L                   Lz\n",
       "0       0     1.2318683862686157    -0.39692866802215576  -0.598057746887207       301.1552734375       174.05947875976562   27.42754554748535    -149431.40625   407.38897705078125  333.9555358886719\n",
       "1       0     3.1000826358795166    -5.2641072273254395   -0.08563928306102753     42.31336975097656    117.65007019042969   -48.13563919067383   -135327.78125   660.099853515625    587.467041015625\n",
       "2       0     2.239809036254883     3.041299819946289     0.20514149963855743      -238.2696533203125   -33.5640754699707    -12.8736572265625    -137704.390625  650.582275390625    649.4723510742188\n",
       "3       0     0.8771036863327026    1.7859472036361694    0.930344820022583        -186.66604614257812  -15.906522750854492  -57.79670715332031   -167917.625     353.51385498046875  319.4240417480469\n",
       "4       0     0.1723162978887558    3.2632522583007812    1.307965874671936        -94.61146545410156   33.2225341796875     -45.685874938964844  -158379.78125   386.50628662109375  314.4658508300781\n",
       "...     ...   ...                   ...                   ...                      ...                  ...                  ...                  ...             ...                 ...\n",
       "10,009  0     0.90480637550354      4.832354545593262     -0.08294452726840973     -96.9912109375       18.704917907714844   -22.30633544921875   -149742.75      497.9065246582031   485.6202392578125\n",
       "10,010  0     0.5669021606445312    1.3759788274765015    0.7074300646781921       -178.99111938476562  292.1639404296875    15.780367851257324   -143447.40625   471.4540100097656   411.9163513183594\n",
       "10,011  0     -0.26697027683258057  2.6641666889190674    0.6911221742630005       -129.47744750976562  -153.0867919921875   -191.64637756347656  -139728.703125  576.6127319335938   385.8191223144531\n",
       "10,012  0     -2.1996543407440186   -0.6662538051605225   -0.00011480561806820333  109.60547637939453   -59.14215087890625   -8.455219268798828   -182279.234375  204.04580688476562  203.1173553466797\n",
       "10,013  0     0.15803414583206177   -3.096275806427002    0.15831845998764038      124.51082611083984   174.66983032226562   -95.10889434814453   -148366.75      493.0278015136719   413.1236572265625"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.id==0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:16:38.683128Z",
     "start_time": "2020-02-24T14:16:37.019989Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:MainThread:vaex:issue loading arrow\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/maartenbreddels/src/vaex/packages/vaex-core/vaex/__init__.py\", line 707, in <module>\n",
      "    add_namespace = entry.load()\n",
      "  File \"/Users/maartenbreddels/miniconda3/envs/dev/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 2443, in load\n",
      "    return self.resolve()\n",
      "  File \"/Users/maartenbreddels/miniconda3/envs/dev/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 2449, in resolve\n",
      "    module = __import__(self.module_name, fromlist=['__name__'], level=0)\n",
      "ModuleNotFoundError: No module named 'vaex.arrow.opener'\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['./pos_b00_relf_nn.dat',\n",
       " './pos_b01_relf_nn.dat',\n",
       " './pos_b02_relf_nn.dat',\n",
       " './pos_b03_relf_nn.dat',\n",
       " './pos_b04_relf_nn.dat',\n",
       " './pos_b05_relf_nn.dat',\n",
       " './pos_b06_relf_nn.dat',\n",
       " './pos_b07_relf_nn.dat',\n",
       " './pos_b08_relf_nn.dat',\n",
       " './pos_b09_relf_nn.dat',\n",
       " './pos_b10_relf_nn.dat',\n",
       " './pos_b11_relf_nn.dat',\n",
       " './pos_b12_relf_nn.dat',\n",
       " './pos_b13_relf_nn.dat',\n",
       " './pos_b14_relf_nn.dat',\n",
       " './pos_b15_relf_nn.dat',\n",
       " './pos_b16_relf_nn.dat',\n",
       " './pos_b17_relf_nn.dat',\n",
       " './pos_b18_relf_nn.dat',\n",
       " './pos_b19_relf_nn.dat',\n",
       " './pos_b20_relf_nn.dat',\n",
       " './pos_b21_relf_nn.dat',\n",
       " './pos_b22_relf_nn.dat',\n",
       " './pos_b23_relf_nn.dat',\n",
       " './pos_b24_relf_nn.dat',\n",
       " './pos_b25_relf_nn.dat',\n",
       " './pos_b26_relf_nn.dat',\n",
       " './pos_b27_relf_nn.dat',\n",
       " './pos_b28_relf_nn.dat',\n",
       " './pos_b29_relf_nn.dat',\n",
       " './pos_b30_relf_nn.dat',\n",
       " './pos_b31_relf_nn.dat',\n",
       " './pos_b32_relf_nn.dat']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import vaex\n",
    "import glob\n",
    "import numpy as np\n",
    "filenames_xyz = glob.glob('./pos_b*_relf_nn.dat')\n",
    "filenames_xyz.sort()\n",
    "filenames_xyz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:16:50.344031Z",
     "start_time": "2020-02-24T14:16:39.379067Z"
    }
   },
   "outputs": [],
   "source": [
    "for i, filename in enumerate(filenames_xyz):\n",
    "    df = vaex.from_ascii(filename, names=['x', 'y', 'z'])\n",
    "    df = df[1:]  # first one is center of mass\n",
    "    df['id'] = np.full(len(df), i, dtype=np.uint8)\n",
    "    df = df[['id', 'x', 'y', 'z']]  # reorder\n",
    "    df.export(f'halo_{i:02}_xyz.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:19:10.325554Z",
     "start_time": "2020-02-24T14:18:59.947231Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "halo_00_vel.hdf5\n",
      "halo_01_vel.hdf5\n",
      "halo_02_vel.hdf5\n",
      "halo_03_vel.hdf5\n",
      "halo_04_vel.hdf5\n",
      "halo_05_vel.hdf5\n",
      "halo_06_vel.hdf5\n",
      "halo_07_vel.hdf5\n",
      "halo_08_vel.hdf5\n",
      "halo_09_vel.hdf5\n",
      "halo_10_vel.hdf5\n",
      "halo_11_vel.hdf5\n",
      "halo_12_vel.hdf5\n",
      "halo_13_vel.hdf5\n",
      "halo_14_vel.hdf5\n",
      "halo_15_vel.hdf5\n",
      "halo_16_vel.hdf5\n",
      "halo_17_vel.hdf5\n",
      "halo_18_vel.hdf5\n",
      "halo_19_vel.hdf5\n",
      "halo_20_vel.hdf5\n",
      "halo_21_vel.hdf5\n",
      "halo_22_vel.hdf5\n",
      "halo_23_vel.hdf5\n",
      "halo_24_vel.hdf5\n",
      "halo_25_vel.hdf5\n",
      "halo_26_vel.hdf5\n",
      "halo_27_vel.hdf5\n",
      "halo_28_vel.hdf5\n",
      "halo_29_vel.hdf5\n",
      "halo_30_vel.hdf5\n",
      "halo_31_vel.hdf5\n",
      "halo_32_vel.hdf5\n"
     ]
    }
   ],
   "source": [
    "filenames_vel = glob.glob('./vel_*.dat')\n",
    "for i, filename in enumerate(filenames_vel):\n",
    "    df = vaex.from_ascii(filename, names=['vx', 'vy', 'vz'])\n",
    "    df = df[1:]  # first one is center of mass\n",
    "    print(f'halo_{i:02}_vel.hdf5')\n",
    "    df.export(f'halo_{i:02}_vel.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:17:52.135788Z",
     "start_time": "2020-02-24T14:17:50.948523Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FeH: -1.48+/-0.15\n",
      "FeH: -1.21+/-0.12\n",
      "FeH: -1.87+/-0.11\n",
      "FeH: -2.29+/-0.11\n",
      "FeH: -2.95+/-0.09\n",
      "FeH: -1.66+/-0.17\n",
      "FeH: -1.69+/-0.18\n",
      "FeH: -2.48+/-0.14\n",
      "FeH: -2.18+/-0.08\n",
      "FeH: -1.63+/-0.10\n",
      "FeH: -1.82+/-0.17\n",
      "FeH: -1.06+/-0.16\n",
      "FeH: -1.03+/-0.11\n",
      "FeH: -2.49+/-0.15\n",
      "FeH: -0.60+/-0.12\n",
      "FeH: -2.16+/-0.10\n",
      "FeH: -1.82+/-0.14\n",
      "FeH: -2.35+/-0.17\n",
      "FeH: -2.07+/-0.15\n",
      "FeH: -1.80+/-0.16\n",
      "FeH: -0.81+/-0.14\n",
      "FeH: -1.11+/-0.12\n",
      "FeH: -1.33+/-0.15\n",
      "FeH: -1.27+/-0.14\n",
      "FeH: -2.72+/-0.09\n",
      "FeH: -1.05+/-0.10\n",
      "FeH: -2.67+/-0.18\n",
      "FeH: -2.46+/-0.10\n",
      "FeH: -2.12+/-0.15\n",
      "FeH: -1.60+/-0.09\n",
      "FeH: -2.00+/-0.15\n",
      "FeH: -1.68+/-0.18\n",
      "FeH: -1.54+/-0.17\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "for i in range(33):\n",
    "    # get rid of header, first row, and footer\n",
    "    data = np.memmap(f'./en_angmom_f_000.{i:02}', np.float32)[4:-1].copy()\n",
    "    E, L, Lz = data.reshape((100_000, 3)).T\n",
    "    FeH_mean = np.random.normal(-1.7, 0.45)\n",
    "    FeH_sigma = 0.13 + 0.05 * (np.random.random() - 0.5)*2\n",
    "    FeH = np.random.normal(FeH_mean, FeH_sigma, len(E))\n",
    "    print(\"FeH: %4.2f+/-%4.2f\" % (FeH_mean, FeH_sigma))\n",
    "    df = vaex.from_arrays(E=E, L=L, Lz=Lz, FeH=FeH)\n",
    "    df.export(f'halo_extra_{i:02}_xyz.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:19:16.468735Z",
     "start_time": "2020-02-24T14:19:16.060122Z"
    }
   },
   "outputs": [],
   "source": [
    "dfs = []\n",
    "for i in range(33):\n",
    "    df1 = vaex.open(f'halo_{i:02}_xyz.hdf5')\n",
    "    df2 = vaex.open(f'halo_{i:02}_vel.hdf5')\n",
    "    df3 = vaex.open(f'halo_extra_{i:02}_xyz.hdf5')\n",
    "    dfs.append(df1.join(df2).join(df3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:19:17.896987Z",
     "start_time": "2020-02-24T14:19:17.870223Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                                 </th><th>id  </th><th>x           </th><th>y          </th><th>z           </th><th>vx         </th><th>vy         </th><th>vz         </th><th>E             </th><th>L                </th><th>Lz                </th><th>FeH                </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>     </td><td>0   </td><td>2.02209926  </td><td>-4.46000433</td><td>1.2478652   </td><td>31.3061771 </td><td>-355.004456</td><td>-126.52121 </td><td>-143642.46875 </td><td>530.4976196289062</td><td>494.7076110839844 </td><td>-1.4976606383212494</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>     </td><td>0   </td><td>-2.81293559 </td><td>2.8778379  </td><td>-1.10256541 </td><td>-346.250732</td><td>52.4870834 </td><td>-164.681458</td><td>-147369.71875 </td><td>486.2104797363281</td><td>455.4730224609375 </td><td>-1.6468179916317884</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>     </td><td>0   </td><td>-5.4028182  </td><td>0.68217665 </td><td>0.0781376138</td><td>0.741490364</td><td>45.1032295 </td><td>30.7795429 </td><td>-135592.3125  </td><td>767.2000122070312</td><td>523.798095703125  </td><td>-1.4276230096474893</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>     </td><td>0   </td><td>1.24775958  </td><td>-1.85724545</td><td>0.25229013  </td><td>18.4966393 </td><td>-54.1744385</td><td>38.6560516 </td><td>-156535.640625</td><td>467.8145446777344</td><td>414.2808532714844 </td><td>-1.4337296742283614</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>     </td><td>0   </td><td>1.64664936  </td><td>4.56083202 </td><td>-0.714936256</td><td>-72.4139557</td><td>34.8387108 </td><td>-81.2754974</td><td>-144798.59375 </td><td>615.1791381835938</td><td>533.921875        </td><td>-1.3216683017018027</td></tr>\n",
       "<tr><td>...                               </td><td>... </td><td>...         </td><td>...        </td><td>...         </td><td>...        </td><td>...        </td><td>...        </td><td>...           </td><td>...              </td><td>...               </td><td>...                </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99,995</i></td><td>0   </td><td>-1.93573666 </td><td>0.490666598</td><td>-0.688795984</td><td>45.4977951 </td><td>7.44010353 </td><td>-38.8004951</td><td>-138716.734375</td><td>620.908203125    </td><td>495.6899719238281 </td><td>-1.302667184567    </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99,996</i></td><td>0   </td><td>-0.608969212</td><td>2.0170064  </td><td>0.0048179375</td><td>-66.9358826</td><td>-112.574234</td><td>-60.1542549</td><td>-155471.390625</td><td>402.1167297363281</td><td>402.0383605957031 </td><td>-1.5063086092266174</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99,997</i></td><td>0   </td><td>-3.67012    </td><td>1.37762702 </td><td>0.908912241 </td><td>-100.639084</td><td>1.37331498 </td><td>-50.4876251</td><td>-156877.5625  </td><td>322.8804016113281</td><td>314.35418701171875</td><td>-1.3630125505495732</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99,998</i></td><td>0   </td><td>1.34276438  </td><td>-2.52453423</td><td>0.926696062 </td><td>11.1622276 </td><td>119.667099 </td><td>-84.9928894</td><td>-133674.21875 </td><td>445.6336975097656</td><td>422.7032165527344 </td><td>-1.4678487048432325</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99,999</i></td><td>0   </td><td>-0.157337546</td><td>-4.32013035</td><td>-0.636850417</td><td>109.544586 </td><td>184.828583 </td><td>146.21405  </td><td>-145261.8125  </td><td>603.845947265625 </td><td>367.0614929199219 </td><td>-1.3574055450240077</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#       id    x             y            z             vx           vy           vz           E               L                  Lz                  FeH\n",
       "0       0     2.02209926    -4.46000433  1.2478652     31.3061771   -355.004456  -126.52121   -143642.46875   530.4976196289062  494.7076110839844   -1.4976606383212494\n",
       "1       0     -2.81293559   2.8778379    -1.10256541   -346.250732  52.4870834   -164.681458  -147369.71875   486.2104797363281  455.4730224609375   -1.6468179916317884\n",
       "2       0     -5.4028182    0.68217665   0.0781376138  0.741490364  45.1032295   30.7795429   -135592.3125    767.2000122070312  523.798095703125    -1.4276230096474893\n",
       "3       0     1.24775958    -1.85724545  0.25229013    18.4966393   -54.1744385  38.6560516   -156535.640625  467.8145446777344  414.2808532714844   -1.4337296742283614\n",
       "4       0     1.64664936    4.56083202   -0.714936256  -72.4139557  34.8387108   -81.2754974  -144798.59375   615.1791381835938  533.921875          -1.3216683017018027\n",
       "...     ...   ...           ...          ...           ...          ...          ...          ...             ...                ...                 ...\n",
       "99,995  0     -1.93573666   0.490666598  -0.688795984  45.4977951   7.44010353   -38.8004951  -138716.734375  620.908203125      495.6899719238281   -1.302667184567\n",
       "99,996  0     -0.608969212  2.0170064    0.0048179375  -66.9358826  -112.574234  -60.1542549  -155471.390625  402.1167297363281  402.0383605957031   -1.5063086092266174\n",
       "99,997  0     -3.67012      1.37762702   0.908912241   -100.639084  1.37331498   -50.4876251  -156877.5625    322.8804016113281  314.35418701171875  -1.3630125505495732\n",
       "99,998  0     1.34276438    -2.52453423  0.926696062   11.1622276   119.667099   -84.9928894  -133674.21875   445.6336975097656  422.7032165527344   -1.4678487048432325\n",
       "99,999  0     -0.157337546  -4.32013035  -0.636850417  109.544586   184.828583   146.21405    -145261.8125    603.845947265625   367.0614929199219   -1.3574055450240077"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:19:40.396352Z",
     "start_time": "2020-02-24T14:19:35.556831Z"
    }
   },
   "outputs": [],
   "source": [
    "df = vaex.concat(dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T14:19:40.492059Z",
     "start_time": "2020-02-24T14:19:40.398389Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                                    </th><th>id  </th><th>x          </th><th>y          </th><th>z           </th><th>vx         </th><th>vy         </th><th>vz         </th><th>E              </th><th>L                 </th><th>Lz                 </th><th>FeH                </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>        </td><td>0   </td><td>2.02209926 </td><td>-4.46000433</td><td>1.2478652   </td><td>31.3061771 </td><td>-355.004456</td><td>-126.52121 </td><td>-143642.46875  </td><td>530.4976196289062 </td><td>494.7076110839844  </td><td>-1.4976606383212494</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>        </td><td>0   </td><td>-2.81293559</td><td>2.8778379  </td><td>-1.10256541 </td><td>-346.250732</td><td>52.4870834 </td><td>-164.681458</td><td>-147369.71875  </td><td>486.2104797363281 </td><td>455.4730224609375  </td><td>-1.6468179916317884</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>        </td><td>0   </td><td>-5.4028182 </td><td>0.68217665 </td><td>0.0781376138</td><td>0.741490364</td><td>45.1032295 </td><td>30.7795429 </td><td>-135592.3125   </td><td>767.2000122070312 </td><td>523.798095703125   </td><td>-1.4276230096474893</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>        </td><td>0   </td><td>1.24775958 </td><td>-1.85724545</td><td>0.25229013  </td><td>18.4966393 </td><td>-54.1744385</td><td>38.6560516 </td><td>-156535.640625 </td><td>467.8145446777344 </td><td>414.2808532714844  </td><td>-1.4337296742283614</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>        </td><td>0   </td><td>1.64664936 </td><td>4.56083202 </td><td>-0.714936256</td><td>-72.4139557</td><td>34.8387108 </td><td>-81.2754974</td><td>-144798.59375  </td><td>615.1791381835938 </td><td>533.921875         </td><td>-1.3216683017018027</td></tr>\n",
       "<tr><td>...                                  </td><td>... </td><td>...        </td><td>...        </td><td>...         </td><td>...        </td><td>...        </td><td>...        </td><td>...            </td><td>...               </td><td>...                </td><td>...                </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,995</i></td><td>32  </td><td>1.60329461 </td><td>3.89954162 </td><td>-3.06202364 </td><td>14.926775  </td><td>91.3861237 </td><td>123.974518 </td><td>-140634.03125  </td><td>223.27337646484375</td><td>-165.4918670654297 </td><td>-1.444779352676424 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,996</i></td><td>32  </td><td>4.44476748 </td><td>3.23454309 </td><td>2.96048737  </td><td>-138.288055</td><td>-132.067291</td><td>-214.071075</td><td>-125076.1875   </td><td>323.4261779785156 </td><td>-283.8592834472656 </td><td>-1.828319322535441 </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,997</i></td><td>32  </td><td>-5.87667322</td><td>-3.19651794</td><td>-1.85363591 </td><td>-74.9682465</td><td>32.2182045 </td><td>107.685356 </td><td>-127123.4765625</td><td>384.3844909667969 </td><td>-329.7651672363281 </td><td>-1.3069962728340068</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,998</i></td><td>32  </td><td>-2.31978846</td><td>5.34516335 </td><td>-2.76076818 </td><td>-208.974823</td><td>19.0502319 </td><td>-218.067688</td><td>-123743.21875  </td><td>425.5377502441406 </td><td>-337.26629638671875</td><td>-1.8696654111490343</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3,299,999</i></td><td>32  </td><td>-2.71977496</td><td>-2.2846334 </td><td>-0.70853442 </td><td>-60.9902077</td><td>-54.8837967</td><td>190.455734 </td><td>-132432.21875  </td><td>455.4076843261719 </td><td>-252.4229278564453 </td><td>-1.3589607540996291</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#          id    x            y            z             vx           vy           vz           E                L                   Lz                   FeH\n",
       "0          0     2.02209926   -4.46000433  1.2478652     31.3061771   -355.004456  -126.52121   -143642.46875    530.4976196289062   494.7076110839844    -1.4976606383212494\n",
       "1          0     -2.81293559  2.8778379    -1.10256541   -346.250732  52.4870834   -164.681458  -147369.71875    486.2104797363281   455.4730224609375    -1.6468179916317884\n",
       "2          0     -5.4028182   0.68217665   0.0781376138  0.741490364  45.1032295   30.7795429   -135592.3125     767.2000122070312   523.798095703125     -1.4276230096474893\n",
       "3          0     1.24775958   -1.85724545  0.25229013    18.4966393   -54.1744385  38.6560516   -156535.640625   467.8145446777344   414.2808532714844    -1.4337296742283614\n",
       "4          0     1.64664936   4.56083202   -0.714936256  -72.4139557  34.8387108   -81.2754974  -144798.59375    615.1791381835938   533.921875           -1.3216683017018027\n",
       "...        ...   ...          ...          ...           ...          ...          ...          ...              ...                 ...                  ...\n",
       "3,299,995  32    1.60329461   3.89954162   -3.06202364   14.926775    91.3861237   123.974518   -140634.03125    223.27337646484375  -165.4918670654297   -1.444779352676424\n",
       "3,299,996  32    4.44476748   3.23454309   2.96048737    -138.288055  -132.067291  -214.071075  -125076.1875     323.4261779785156   -283.8592834472656   -1.828319322535441\n",
       "3,299,997  32    -5.87667322  -3.19651794  -1.85363591   -74.9682465  32.2182045   107.685356   -127123.4765625  384.3844909667969   -329.7651672363281   -1.3069962728340068\n",
       "3,299,998  32    -2.31978846  5.34516335   -2.76076818   -208.974823  19.0502319   -218.067688  -123743.21875    425.5377502441406   -337.26629638671875  -1.8696654111490343\n",
       "3,299,999  32    -2.71977496  -2.2846334   -0.70853442   -60.9902077  -54.8837967  190.455734   -132432.21875    455.4076843261719   -252.4229278564453   -1.3589607540996291"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-24T13:41:57.238441Z",
     "start_time": "2020-02-24T13:41:57.234589Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.1444091796875"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "100_000 * 4 * 3 / 1024**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
